# Updated by Wei on 20140629
# Consider NOT doing the AND evaluation, cause OR is good in low % index kept level
exit(1)

from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

# Note:
print "20140402Night: NO fallThroughRate output files have been implemented. I need to add that after I use it"
exit(1)

# for pangolin:
# inputFileName0 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/data/realFreqOfTermsForTail5KFrom100K"
# for dodo:
# inputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/realFreqOfTermsForTail5KFrom100K"
# for moa:
inputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/realFreqOfTermsForTail5KFrom100K"

# AND
# for pangolin:
# inputFileName1 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/data/rawResults_uniform_impactScore_listLength_100%_TOP10_AND_20131210Night"
# for dodo:
# inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/rawResults_uniform_impactScore_listLength_100%_TOP10_AND_20131210Night"
# for moa:
inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/rawResults_uniform_impactScore_listLength_100%_TOP10_AND_20131210Night"

# for pangolin:
# inputFileName2 = "/data/obukai/gov2ClearYourMindAndDoItAgain2014/partOfLexiconTermsWithTermIDForTail5KQueries"
# for dodo:
# inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/partOfLexiconTermsWithTermIDForTail5KQueries"
# for moa:
inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/partOfLexiconTermsWithTermIDForTail5KQueries"

# options:
# for pangolin:
# inputFileName3 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140321Morning_weight_0_WHOLE"
# for dodo:
# inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140321Morning_weight_0_WHOLE"
# inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140321Morning_weight_10_WHOLE"
# inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140321Morning_weight_100_WHOLE"
# for moa:
inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/allPostingsBeingPopped20140402Morning_weight_0_SPECIAL_Pt_1"

# for dodo:
# for pangolin
# inputFileName4 = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/data/queryTermsWithTheirLengthsOfInvertedList"
# for moa:
inputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-April/data/queryTermsWithTheirLengthsOfInvertedList"

def getQueryIDAndResultSetGivenResultFileForGoldStandardResultFile(inputFileName1,NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE,termANDTermIDDict,searchSystemDocumentResultDict,postingWithDocumentResultDict,searchSystemDocumentResultDictNumOfPostingsContributedCurrentlyDict,searchSystemDocumentResultWithNumOfPostingsContributedDict):
    print "getQueryIDAndResultSetGivenResultFileForGoldStandardResultFile() function begins..."
    #tempQueryTermDict = {}
    #tempOutputFileName = "/data/obukai/gov2ClearYourMindAndDoItAgain2014/partOfLexiconTermsWithTermIDForTail5KQueries"
    #tempOutputFileHandler = open(tempOutputFileName,"w")
    
    tempCounter = 0
    tempCounter2 = 0
    numOfQueriesHavingQID = 0
    numOfQueriesHavingSearchContent = 0
    numOfQueriesHavingSearchResults = 0
    
    inputFileHandler1 = open(inputFileName1,"r")
    
    
    currentLine = inputFileHandler1.readline()
    nextLine = ""
    currentQID = ""
    
    numOfResultsForTheAllQueries = 0
    
    while currentLine:
        # sampling parsing line:
        # qid: 701
        if currentLine.startswith("qid:"):
            currentQID = currentLine.strip().split(":")[1].strip()
            print "currentQID:",currentQID
            numOfQueriesHavingQID += 1
        
        # sample parsing line:
        # Search: u s  oil industry history
        if currentLine.startswith("Search:"):
            elementList = currentLine.strip().split(" ")
            
            currentSearchContent = ""
            
            queryTermList = []
            for element in elementList[1:]:
                if element.strip() != "":
                    queryTermList.append( element.strip() )        
            currentQueryLength = len(queryTermList)
            print "currentQueryLength:",currentQueryLength
    
            for element in queryTermList:
                currentSearchContent += element + " "
            
            print currentSearchContent
            numOfQueriesHavingSearchContent += 1
            
            
            currentQueryTermIndexDict = {}
            nextLine = inputFileHandler1.readline()
            # print nextLine.strip()
            parsedFlag = True
            
            if nextLine.strip().endswith("is NOT in the lexicon."):
                parsedFlag = False
            
            if nextLine.strip() != "" and parsedFlag:
                
                # skip the following sampling line: 
                # [2013-05-24 14:20:44] 38871 terms with their lists length have been loaded.
                # nextLine = inputFileHandler1.readline()
                
                nextLineElements = nextLine.strip().split(" ")
                # print "nextLineElements:",nextLineElements
                
                # sample parsing line: 
                # oil:0 industry:1 history:2 u:3 s:4
                for element in nextLineElements:
                    if element.split(":")[0] not in currentQueryTermIndexDict:
                        currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
                
                print "currentQueryTermIndexDict:",currentQueryTermIndexDict
                # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
                
                # temp section
                #for queryTermIndex in currentQueryTermIndexDict:
                #    currentQueryTerm = currentQueryTermIndexDict[queryTermIndex]
                #    if currentQueryTerm not in tempQueryTermDict:
                #        tempQueryTermDict[currentQueryTerm] = 1
                #        tempOutputFileHandler.write( currentQueryTerm + "\n")
                
                
                # Now, it is time to read the threshold line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                    
                # Now, it is time to read computation method line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                
                # skip one line
                currentLine = inputFileHandler1.readline()
                
                # Now, it is time to read the actual training example line
                currentLine = inputFileHandler1.readline()
                numOfResultsForTheCurrentQuery = 0
                
                # for IMPORTANT DEBUG ONLY
                #print "len( currentLine.strip().split(" ") ):",len( currentLine.strip().split(" ") )
                #print "currentLine:",currentLine
                #exit(1)
                lineElements = currentLine.strip().split(" ")
                while len( lineElements ) == 14:
                    theRank = int(lineElements[0])
                    theDocID = lineElements[-2]
                    documentResultKey = currentQID + "_" + theDocID
                    if currentQID not in searchSystemDocumentResultDictNumOfPostingsContributedCurrentlyDict:
                        searchSystemDocumentResultDictNumOfPostingsContributedCurrentlyDict[currentQID] = {}
                        searchSystemDocumentResultWithNumOfPostingsContributedDict[currentQID] = {}
                    
                    if currentQID not in searchSystemDocumentResultDict:
                        searchSystemDocumentResultDict[currentQID] = {}
                    
                    if theRank <= 10:
                        if documentResultKey not in searchSystemDocumentResultDict:
                            searchSystemDocumentResultDict[currentQID][documentResultKey] = 0.0
                        else:
                            print "duplicated document result."
                            print "documentResultKey:",documentResultKey
                            exit(1)
                        
                        
                        searchSystemDocumentResultDictNumOfPostingsContributedCurrentlyDict[currentQID][documentResultKey] = 0
                        searchSystemDocumentResultWithNumOfPostingsContributedDict[currentQID][documentResultKey] = 0
                        
                        upperBound = 0
                        if len(currentQueryTermIndexDict) >= 10:
                            upperBound = 10
                        else:
                            upperBound = len(currentQueryTermIndexDict)
                        
                        for i in range(0,upperBound):
                            currentTerm = currentQueryTermIndexDict[i]
                            if currentTerm not in termANDTermIDDict:
                                print currentTerm,"is NOT in the dict."
                                exit(1)
                            else:
                                currentTermID = termANDTermIDDict[currentTerm]
                            currentTermScore = float( lineElements[1 + i] )
                            if currentTermScore != 0.0:
                                postingKey = currentTermID + "_" + theDocID
                                if postingKey not in postingWithDocumentResultDict:
                                    postingWithDocumentResultDict[postingKey] = []
                                
                                postingWithDocumentResultDict[postingKey].append(documentResultKey)    
                                searchSystemDocumentResultWithNumOfPostingsContributedDict[currentQID][documentResultKey] += 1
                                tempCounter += 1
                        tempCounter2 += len(currentQueryTermIndexDict)
                    else:
                        pass
                    


                    numOfResultsForTheCurrentQuery += 1
                    
                    currentLine = inputFileHandler1.readline()
                    lineElements = currentLine.strip().split(" ")
                   
                numOfResultsForTheAllQueries += numOfResultsForTheCurrentQuery
                
                # sampling method 1: among all the top10 documents, select those documents, open those documents and extract the corresponding postings.
                # This method make sure that once the document have been selected, ALL the coresponding postings will also be selected.
                # This sampling method has been developed on 2013/01/09
                
                print "numOfResultsForTheCurrentQuery:",numOfResultsForTheCurrentQuery
                print "tempCounter:",tempCounter
                print "tempCounter2:",tempCounter2
                print
            
        currentLine = inputFileHandler1.readline()
        
    print "numOfQueriesHavingQID:",numOfQueriesHavingQID
    print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
    print "numOfQueriesHavingSearchResults:",numOfQueriesHavingSearchResults
    print "getQueryIDAndResultSetGivenResultFileForGoldStandardResultFile() function ends."




# key: qID_docID
# value: 1
searchSystemDocumentResultDict = {}
searchSystemDocumentResultDictNumOfPostingsContributedCurrentlyDict = {}
searchSystemDocumentResultWithNumOfPostingsContributedDict = {}
postingWithDocumentResultDict = {}


termANDTermIDDict = {}


inputFileHandler = open(inputFileName2,"r")
currentLine = inputFileHandler.readline()
currentLineNum = 0

while currentLine:
    if currentLineNum % 1000000 == 0:
        print currentLineNum,"lines loaded."
    currentLineElements = currentLine.strip().split(" ")
    currentTermID = currentLineElements[0]
    currentTerm = currentLineElements[1]
    if currentTerm not in termANDTermIDDict:
        termANDTermIDDict[currentTerm] = currentTermID
    currentLine = inputFileHandler.readline()
    currentLineNum += 1
print "len(termANDTermIDDict): ",len(termANDTermIDDict)
inputFileHandler.close()


getQueryIDAndResultSetGivenResultFileForGoldStandardResultFile(inputFileName1,10,termANDTermIDDict,searchSystemDocumentResultDict,postingWithDocumentResultDict,searchSystemDocumentResultDictNumOfPostingsContributedCurrentlyDict,searchSystemDocumentResultWithNumOfPostingsContributedDict)
print "overall:"

documentResultCounter = 0
for currentQID in searchSystemDocumentResultDict:
    documentResultCounter += len( searchSystemDocumentResultDict[currentQID] )
print "# of top10 document results:",documentResultCounter

postingCounter = 0
for currentPostingKey in postingWithDocumentResultDict:
    postingCounter += len(postingWithDocumentResultDict[currentPostingKey])
print "# of top10 postings:",postingCounter

# key: term
# value: freq
queryTermDict = {}

inputFileHandler = open(inputFileName0,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentQueryTerm = lineElements[0]
    currentQueryTermFreq = int(lineElements[1])
    if currentQueryTerm not in queryTermDict:
        queryTermDict[currentQueryTerm] = currentQueryTermFreq
print "len(queryTermDict):",len(queryTermDict)
print "queryTermDict['you']:",queryTermDict['you']
inputFileHandler.close()

queryTermWithTheirLengthOfInvertedListDict = {}
inputFileHandler = open(inputFileName4,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    lengthOfList = int(lineElements[1])
    if queryTerm not in queryTermWithTheirLengthOfInvertedListDict:
        queryTermWithTheirLengthOfInvertedListDict[queryTerm] = lengthOfList
print "len(queryTermWithTheirLengthOfInvertedListDict):",len(queryTermWithTheirLengthOfInvertedListDict)
print "queryTermWithTheirLengthOfInvertedListDict['you']:",queryTermWithTheirLengthOfInvertedListDict['you']
inputFileHandler.close()

#################################################################################################################
TOTAL_NUM_OF_POSTINGS = 6451948010
numOfPostingNeededToBePoppedAtDebugPercentage = 5000000
numOfPostingNeededToBePoppedAt1Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.01)
numOfPostingNeededToBePoppedAt3Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.03)
numOfPostingNeededToBePoppedAt5Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.05)
numOfPostingNeededToBePoppedAt10Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.1)
numOfPostingNeededToBePoppedAt15Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.15)
numOfPostingNeededToBePoppedAt20Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.2)
numOfPostingNeededToBePoppedAt30Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.3)
numOfPostingNeededToBePoppedAt40Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.4)
numOfPostingNeededToBePoppedAt50Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.5)
numOfPostingNeededToBePoppedAt60Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.6)
numOfPostingNeededToBePoppedAt70Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.7)
numOfPostingNeededToBePoppedAt80Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.8)
numOfPostingNeededToBePoppedAt90Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.9)
print "numOfPostingNeededToBePoppedAtDebugPercentage:",numOfPostingNeededToBePoppedAtDebugPercentage
print "numOfPostingNeededToBePoppedAt1Percentage:",numOfPostingNeededToBePoppedAt1Percentage
print "numOfPostingNeededToBePoppedAt3Percentage:",numOfPostingNeededToBePoppedAt3Percentage
print "numOfPostingNeededToBePoppedAt5Percentage:",numOfPostingNeededToBePoppedAt5Percentage
print "numOfPostingNeededToBePoppedAt10Percentage:",numOfPostingNeededToBePoppedAt10Percentage
print "numOfPostingNeededToBePoppedAt15Percentage:",numOfPostingNeededToBePoppedAt15Percentage
print "numOfPostingNeededToBePoppedAt20Percentage:",numOfPostingNeededToBePoppedAt20Percentage
print "numOfPostingNeededToBePoppedAt30Percentage:",numOfPostingNeededToBePoppedAt30Percentage
print "numOfPostingNeededToBePoppedAt40Percentage:",numOfPostingNeededToBePoppedAt40Percentage
print "numOfPostingNeededToBePoppedAt50Percentage:",numOfPostingNeededToBePoppedAt50Percentage
print "numOfPostingNeededToBePoppedAt60Percentage:",numOfPostingNeededToBePoppedAt60Percentage
print "numOfPostingNeededToBePoppedAt70Percentage:",numOfPostingNeededToBePoppedAt70Percentage
print "numOfPostingNeededToBePoppedAt80Percentage:",numOfPostingNeededToBePoppedAt80Percentage
print "numOfPostingNeededToBePoppedAt90Percentage:",numOfPostingNeededToBePoppedAt90Percentage
numOfTOP10PostingsRetained = 0
numOfTop10DocumentResultsRetained = 0
numOfBytes = 0
numOfPostingPopped = 0
numOfPostingBeInTOP10 = 0

inputFileHandler = open(inputFileName3,"rb")

print "inputFileName1:",inputFileName1
print "inputFileName2:",inputFileName2
print "inputFileName3:",inputFileName3
print "--->posting file to evaluate:",inputFileName3
statinfo = os.stat(inputFileName3)


fileSize = statinfo.st_size
print "file size:",fileSize
numOfResultsReturnedCountedUpTo10 = 0

# value:
# termID
# key:
# # of posting in the list length
termIDAndListLengthDict = {}

while numOfBytes < fileSize:
    # old version
    # each time, just read the info of ONE posting, too few
    # byteString = inputFileHandler.read(4 + 4 + 4)
    # (termID,docID,currentProbability) = unpack( "2I1f", byteString)
    
    # current version
    # each time, read the info of 1M postings
    byteStringBuffer = inputFileHandler.read( 1000000 * 16)
    byteStringBufferIndexPosition = 0
    for i in range(0,1000000):
        byteString = byteStringBuffer[byteStringBufferIndexPosition:byteStringBufferIndexPosition+16]
        byteStringBufferIndexPosition += 16
        (termID,docID,currentProbability,impactScore) = unpack( "2I2f", byteString)
        postingKey = str(termID) + "_" + str(docID)
        
        if str(termID) not in termIDAndListLengthDict:
            termIDAndListLengthDict[str(termID)] = 1
        else:
            termIDAndListLengthDict[str(termID)] += 1
        
        if postingKey in postingWithDocumentResultDict:
            numOfTOP10PostingsRetained += len(postingWithDocumentResultDict[postingKey])
            
        # debug
        # print "postingKey:",postingKey
        # print termID,docID,currentProbability,impactScore
        if postingKey in postingWithDocumentResultDict:
            # interesting issue solved
            # print "touched."
            for currentDocumentResultKey in postingWithDocumentResultDict[postingKey]:
                currentQID = currentDocumentResultKey.strip().split("_")[0]
                searchSystemDocumentResultDictNumOfPostingsContributedCurrentlyDict[currentQID][currentDocumentResultKey] += 1
                # tempCounter += 1

        if numOfPostingPopped == numOfPostingNeededToBePoppedAtDebugPercentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt1Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt3Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt5Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt10Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt15Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt20Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt30Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt40Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt50Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt60Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt70Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt80Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt90Percentage:
            
            numOfRelatedDocumentResultReturned = 0
            
            for currentQID in searchSystemDocumentResultDict:
                for currentDocumentResultKey in searchSystemDocumentResultDict[currentQID]:
                    if searchSystemDocumentResultDictNumOfPostingsContributedCurrentlyDict[currentQID][currentDocumentResultKey] == searchSystemDocumentResultWithNumOfPostingsContributedDict[currentQID][currentDocumentResultKey]:
                        numOfRelatedDocumentResultReturned += 1

            print "numOfRelatedDocumentResultReturned:",numOfRelatedDocumentResultReturned
            print "numOfPostingPopped:",numOfPostingPopped
            print "numOfTOP10PostingsRetained:",numOfTOP10PostingsRetained
            print "len(termIDAndListLengthDict):",len(termIDAndListLengthDict)
            
            evaluationCostForCPU = 0
            for currentQueryTerm in queryTermDict:
                if currentQueryTerm in termANDTermIDDict:
                    currentQueryTermID = termANDTermIDDict[currentQueryTerm]
                    if currentQueryTermID in termIDAndListLengthDict:
                        evaluationCostForCPU += termIDAndListLengthDict[currentQueryTermID] * queryTermDict[currentQueryTerm]
            print "evaluationCostForCPU:",evaluationCostForCPU
            
            
        numOfBytes += 12
        numOfPostingPopped += 1
        if numOfPostingPopped % 1000000 == 0:
            print str(numOfPostingPopped),"postings have been examined."

inputFileHandler.close()

print "Note: This process will NEVER finsih and will have an error"
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "inputFileName2:",inputFileName2
print "inputFileName3:",inputFileName3
print "inputFileName4:",inputFileName4
exit(1)



